#!/bin/sh
BASE_HOME=`/usr/bin/dirname "$0"`
FILE_NAME=`basename "$0"`
PROCESS_ID="$$"
CONF_FILE="${BASE_HOME}/mysql_config.list"
##load mysql config file 
while read line; do
    eval $line
done < ${CONF_FILE}


#create logfile path if not exists
LOGFILE_PATH=`echo ${LOGFILE%/*}`
if [ ! -d ${LOGFILE_PATH} ]; then
                mkdir -p  ${LOGFILE_PATH} 
fi

##record scripts running logs
exec >> ${LOGFILE} 2>&1
#slave 
slaveIP=${REMOTE_HOST}
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]keepaliaved into fault status and switch mysql from master node ${LOCAL_HOST} to slave node ${REMOTE_HOST}"
##if this scripts is running and exit
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]check scripts ps -ef | grep \"$0\" | grep -v ${PROCESS_ID} | grep -v grep is running "
check_running=`ps -ef | grep "$0" | grep -v ${PROCESS_ID} | grep -v grep |wc -l`
ps -ef | grep "$0" | grep -v ${PROCESS_ID} | grep -v grep
if [ ${check_running} -ge 1 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[warn]scripts $0 is running ${check_running} and exit"
                #exit 0
else
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]scripts $0 is not running ${check_running} and continue to switch"
fi 

# mysql login CMD
MYSQL_L_CMD="${MYSQL_CMD} -u${SLAVE_USR} -p${SLAVE_PWD} -P${SLAVE_PORT} -h${REMOTE_HOST} --protocol=tcp"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]MYSQL_L_CMD is ${MYSQL_L_CMD}"




BINLOG_PATH=`echo ${LOCAL_BINLOG%/*}`
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]local mysql node binlog path is ${BINLOG_PATH}"

##if the write vip in this node and this node is master node
WVIP_COUNT=`ip addr | grep -v grep | grep "${WRITE_VIP}" |wc -l`
if [ ${WVIP_COUNT} -eq 1 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]this mysql node ${LOCAL_HOST} is master and continue to switch"
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]VIP `ip addr | grep -v grep | grep \"${WRITE_VIP}\"`"
else
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]VIP ${WRITE_VIP} not exists on local host and  mysql node is slave,not need to switch"
    #exit 0
fi

#create binlog parse to sql work dir if not exists
if [ ! -d ${sqlpath} ]; then
                mkdir -p  ${sqlpath} 
fi


#clear the files
echo "" >${sqlpath}/sync.sql
echo "" >${sqlpath}/sync_file
echo "" >${sqlpath}/tmp.txt


LAST_BLOGFILE=`ls -l ${LOCAL_BINLOG}.* | grep -v "index" |sed -n '$p'  | awk '{print $NF}' `
LAST_BLOGFILEN=`echo ${LAST_BLOGFILE} |awk -F"/" '{print $NF}'`
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]local mysql last binlog file is ${LAST_BLOGFILE} and last binlog number is ${LAST_BLOGFILEN}"




# show slave master status
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]show slave master status"
${MYSQL_L_CMD} -e "show master status \G" > ${sqlpath}/master.txt
cat ${sqlpath}/master.txt

getstatus(){

# slave status
${MYSQL_L_CMD} -e "show slave status \G" > ${sqlpath}/tmp.txt

# Master_Log_File
READFILE=`cat ${sqlpath}/tmp.txt |grep -w "Master_Log_File"|grep -v "Relay"|awk -F": " '{print $2}'`

#Relay_Master_Log_File
EXECFILE=`cat ${sqlpath}/tmp.txt |grep -w "Relay_Master_Log_File"|grep "Relay"|awk -F": " '{print $2}'`

# Exec_Master_Log_Pos
EXECPOS=`cat ${sqlpath}/tmp.txt | grep -w 'Exec_Master_Log_Pos' | awk -F ': ' '{print $2}'`

# Read_Master_Log_Pos
READPOS=`cat ${sqlpath}/tmp.txt | grep -w "Read_Master_Log_Pos" | awk -F ': ' '{print $2}'`

#get master binlog endpos
MASTERENDPOS=`${MYSQLBINLOG_CMD} ${LAST_BLOGFILE} |  grep 'end_log_pos' | sed -n '$p' | awk -F ' ' '{print $7}'`

echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]master_binlogfile_endpos:${MASTERENDPOS}"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_readpos:${READPOS}"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_execpos:${EXECPOS}"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_readfile:${READFILE}"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_execfile:${EXECFILE}"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]master_blogfile:${LAST_BLOGFILEN}"
}

getstatus

returnstatus(){

echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_readfile=${READFILE} != slave_execfile=${EXECFILE} or slave_readpos=${READPOS} != slave_execpos=${EXECPOS},waiting 5s for SQL_thread to catch up with IO_thread"
sleep 5
getstatus

}

#get slave status of sql_thread
SQL_STATUS="`grep -w  \"Slave_SQL_Running\" ${sqlpath}/tmp.txt|awk -F\": \" '{print $2}'`"
if [ "${SQL_STATUS}"x != ""x ]; then
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]Slave_SQL_Running:${SQL_STATUS}"
else
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[warn]local mysql ${LOCAL_HOST} is slave and Slave_SQL_Running:${SQL_STATUS}"
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[warn]not need to switch and the local node is slave and exit fault scripts running"
    exit 1
fi

##set remote mysql read_only
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]set remote host ${REMOTE_HOST} mysql read_only=on and super_read_only=on"
${MYSQL_L_CMD} -N -L -s -e "set  global super_read_only=ON;set global read_only=ON;show variables like '%read_only';"

##use mysqlbinlog to createed slave io_thread is not sync master binlog file and parse to sql file
if [ ${MASTERENDPOS} == ${READPOS} ] && [ ${LAST_BLOGFILEN} == ${READFILE} ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]the slave_io_thread is not delayed"
elif [ ${MASTERENDPOS} != ${READPOS} ] && [ ${LAST_BLOGFILEN} == ${READFILE} ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave node ${REMOTE_HOST} not sync completed from master node ${LOCAL_HOST} and started parse master node binlog file ${MYSQLBINLOG_CMD} --start-position  ${READPOS} ${LAST_BLOGFILE} > ${sqlpath}/sync.sql"
    ${MYSQLBINLOG_CMD} --start-position  ${READPOS} ${LAST_BLOGFILE} > ${sqlpath}/sync.sql
    if [ $? -eq 0 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${sqlpath}/sync.sql completed"
                ls -l ${sqlpath}/sync.sql
    else
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${sqlpath}/sync.sql failed and exit the mysql switch\G"
        exit 1
    fi
else [ ${MASTERENDPOS} != ${READPOS} ] && [ ${LAST_BLOGFILEN} != ${READFILE} ]
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave node ${REMOTE_HOST} not sync completed from master node ${LOCAL_HOST} and started parse master node binlog file ${MYSQLBINLOG_CMD} --start-position  ${READPOS}  ${BINLOG_PATH}/${READFILE} >${sqlpath}/sync.sql"   
    ${MYSQLBINLOG_CMD} --start-position  ${READPOS}  ${BINLOG_PATH}/${READFILE} >${sqlpath}/sync.sql
    if [ $? -eq 0 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${sqlpath}/sync.sql completed"
                ls -l ${sqlpath}/sync.sql
    else
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${sqlpath}/sync.sql failed and exit the mysql switch\G"
        exit 1
    fi    
    count=`cat ${LOCAL_BINLOG}.index|wc -l`
    grep -A $count ${READFILE} ${LOCAL_BINLOG}.index|grep -v ${READFILE} > ${sqlpath}/sync_file
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]need to mysqlbinlog to parse binlogfile list"
    cat ${sqlpath}/sync_file
    for i in `cat ${sqlpath}/sync_file `
    do
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${MYSQLBINLOG_CMD}   ${i} >>${sqlpath}/sync.sql start"
        ${MYSQLBINLOG_CMD}   ${i} >>${sqlpath}/sync.sql
        if [ $? -eq 0 ]; then
                                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${MYSQLBINLOG_CMD}   ${i} >>${sqlpath}/sync.sql completed"
                                ls -l ${sqlpath}/sync.sql
                else
                    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]mysqlbinlog write to ${MYSQLBINLOG_CMD}   ${i} >>${sqlpath}/sync.sql failed and exit the mysql switch\G"
                    exit 1
                fi
    done

fi



##if slave sql_thread is not start and started
if [ "${SQL_STATUS}"x == "Yes"x ];then
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]Slave_SQL_thread is running :${SQL_STATUS}"
else
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]Slave_SQL_thread is not running :${SQL_STATUS},will start slave sql_thread" 
    ${MYSQL_L_CMD} -e "start slave sql_thread;"
    if [ $? -eq 0 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_SQL_thread  started sucusessfull and show slave status\G"
                ${MYSQL_L_CMD} -e "show slave status \G;"
    else
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave_SQL_thread  started failed and exit the mysql switch\G"
        exit 1
    fi
    
fi


##if slave sql_thread apply all io_thread  and catch up master by execute sql file
#if slave sql_thread not apply all io_thread and continue to apply until sql_thread applied all io_thread
if [ ${READFILE} == ${EXECFILE} ] && [ ${READPOS} == ${EXECPOS} ];then
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave sql thread is catched up slave_readfile=${READFILE} == slave_execfile=${EXECFILE} ] && [ slave_readpost=${READPOS} == slave_execpos=${EXECPOS}"
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]set global super_read_only = OFF and start catch up master binlog file sql ${sqlpath}/sync.sql " 
    ${MYSQL_L_CMD} -N -L -s -e "set global super_read_only = OFF;set global read_only=ON;show variables like '%read_only';"
    echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]start execute ${sqlpath}/sync.sql"
    ls -l ${sqlpath}/sync.sql
    ${MYSQL_L_CMD}<${sqlpath}/sync.sql
    if [ $? -eq 0 ]; then
                echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]execute ${sqlpath}/sync.sql completed"
    else
        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]execute ${sqlpath}/sync.sql failed and exit mysql switch"
        exit 1
    fi
else
    i=0
    while [ ${READFILE} != ${EXECFILE} ] || [ ${READPOS} != ${EXECPOS} ]
    do
      echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]start ${i} times sleep 5 senconds for slave sql thread to catch up slave_readfile=${READFILE} != slave_execfile=${EXECFILE} ] || [ slave_readpos=${READPOS} != slave_execpos=${EXECPOS}"
      returnstatus
      i=$(($i+1))
      if [ ${READFILE} == ${EXECFILE} ] && [ ${READPOS} == ${EXECPOS} ]; then
          echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]slave sql thread is catched up slave_readfile=${READFILE} == slave_execfile=${EXECFILE} ] && [ slave_readpos=${READPOS} == slave_execpos=${EXECPOS}"
          echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]set global super_read_only = OFF and start catch up master binlog by sql file `ls -l ${sqlpath}/sync.sql` " 
          ${MYSQL_L_CMD} -N -L -s -e "set global super_read_only = OFF;set global read_only=ON;show variables like '%read_only';"
          ${MYSQL_L_CMD}<${sqlpath}/sync.sql 
              if [ $? -eq 0 ]; then
                                        echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]execute ${sqlpath}/sync.sql completed"
              else
                  echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]execute ${sqlpath}/sync.sql failed and exit mysql switch"
                  exit 1
              fi
          echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]set global super_read_only = OFF and  catch up master binlog  by sql file completed" 
          break
      else
          echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]had ${i} times sleep 5 senconds for slave sql thread to catch up slave_readfile=${READFILE} != slave_execfile=${EXECFILE} ] || [ slave_readpos=${READPOS} != slave_execpos=${EXECPOS}"
          #sleep 5
      fi
    done
fi


##backuped the temp file
dates=`date +%Y%m%d%H%M%S`
cp -a ${sqlpath}/sync.sql  ${sqlpath}/sync.sql_${dates}
cp -a ${sqlpath}/tmp.txt   ${sqlpath}/tmp.txt_${dates}
cp -a ${sqlpath}/sync_file ${sqlpath}/sync_file_${dates}

##set slave node as master role
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]start set slave ${REMOTE_HOST} as master role"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]set remote host ${REMOTE_HOST} mysql read_only=on and super_read_only=off"
${MYSQL_L_CMD} -N -L -s -e "set  global super_read_only=OFF;set global read_only=ON;show variables like '%read_only';"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]stop slave; reset slave;reset slave all;"
${MYSQL_L_CMD} -e 'stop slave; reset slave;reset slave all;'
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]show master status, super_read_only = ON; set global read_only = OFF "
${MYSQL_L_CMD} -e 'show master status \G;set global super_read_only = ON; set global read_only = OFF;'
${MYSQL_L_CMD} -N -L -s -e "show variables like 'super_read_only';show variables like 'read_only';"
${MYSQL_L_CMD} -e "show slave status \G"
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]completed set slave ${REMOTE_HOST} as master role"

##check the vip status 
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]check the vip status "
ps -ef | grep -i keepalived | grep -v grep 
ip addr 
#/usr/bin/pkill keepalived 

echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]check the vip status secondary " 
ps -ef | grep -i keepalived | grep -v grep 
ip addr
echo "`date +'%F %T'`|${FILE_NAME}|${PROCESS_ID}|[info]Done successfully switch from ${LOCAL_HOST} to ${REMOTE_HOST}"
